import torch
from torch import nn
import matplotlib.pyplot as plt
import numpy as np
import torch.nn.functional as F
import random
import rl_utils
import gym
from tqdm import tqdm
import env


def dis_to_con(discrete_action, env, action_dim):  # 离散动作转回连续的函数
    action_lowbound = env.action_space.low[0]  # 连续动作的最小值
    action_upbound = env.action_space.high[0]  # 连续动作的最大值
    return action_lowbound + (discrete_action /
                              (action_dim - 1)) * (action_upbound -
                                                   action_lowbound)


class Qnet(torch.nn.Module):
    ''' 只有一层隐藏层的Q网络 '''

    def __init__(self, state_dim, hidden_dim, action_dim):
        super(Qnet, self).__init__()
        self.fc1 = torch.nn.Linear(state_dim, hidden_dim)
        self.fc2 = torch.nn.Linear(hidden_dim, action_dim)

    def forward(self, x):
        x = F.relu(self.fc1(x))
        return self.fc2(x)


class VAnet(torch.nn.Module):
    ''' 只有一层隐藏层的A网络和V网络 '''

    def __init__(self, state_dim, hidden_dim, action_dim):
        super(VAnet, self).__init__()
        self.fc1 = torch.nn.Linear(state_dim, hidden_dim)  # 共享网络部分
        self.fc_A = torch.nn.Linear(hidden_dim, action_dim)
        self.fc_V = torch.nn.Linear(hidden_dim, 1)

    def forward(self, x):  # x:(1,21)
        A = self.fc_A(F.relu(self.fc1(x)))
        V = self.fc_V(F.relu(self.fc1(x)))
        Q = V + A - A.mean(1).view(-1, 1)  # Q值由V值和A值计算得到
        return Q


def train_DQN(agent, env, num_episodes, replay_buffer, minimal_size,
              batch_size):
    return_list = []
    max_q_value_list = []
    max_q_value = 0
    for i in range(10):
        with tqdm(total=int(num_episodes / 10),
                  desc='Iteration %d' % i) as pbar:
            for i_episode in range(int(num_episodes / 10)):
                episode_return = 0
                state, cap = env.reset()  # state (3,) now state (1,3,21)
                done = False
                for vec_num in range(3):
                    while not done:
                        action = agent.take_action(state[0][vec_num])  # action: int i:vec_num
                        max_q_value = agent.max_q_value(
                            state[0][vec_num]) * 0.005 + max_q_value * 0.995  # max_q_value: int  # 平滑处理
                        max_q_value_list.append(max_q_value)  # 保存每个状态的最大Q值
                        # action_continuous = dis_to_con(action, env,
                        #                                21)  # action_continuous: float64
                        next_state, reward, done, cap = env.step(
                            [[vec_num]], [[action]])  # next_state: (3,) reward: int, done: bool
                        # next_state = next_state.view(64,63)
                        replay_buffer.add(state, action, reward, next_state, done)
                        # print(reward)
                        state = next_state  # state : (3, )
                        episode_return += reward
                        if replay_buffer.size() > minimal_size:
                            b_s, b_a, b_r, b_ns, b_d = replay_buffer.sample(
                                batch_size)  # b_a: (64), b_d: (31), b_ns: (64,3), b_r: (64) b_s(64,3)
                            transition_dict = {
                                'states': b_s,
                                'actions': b_a,
                                'next_states': b_ns,
                                'rewards': b_r,
                                'dones': b_d
                            }
                            agent.update(transition_dict)

                return_list.append(episode_return)
                if (i_episode + 1) % 10 == 0:
                    pbar.set_postfix({
                        'episode':
                            '%d' % (num_episodes / 10 * i + i_episode + 1),
                        'return':
                            '%.3f' % np.mean(return_list[-10:])
                    })
                pbar.update(1)
    return return_list, max_q_value_list


class DQN:
    """
        DQN算法,包括Double DQN和Dueling DQN
    """

    def __init__(self,
                 state_dim,
                 hidden_dim,
                 action_dim,
                 learning_rate,
                 gamma,
                 epsilon,
                 target_update,
                 device,
                 dqn_type='VanillaDQN'):
        self.action_dim = action_dim
        if dqn_type == 'DuelingDQN':  # Dueling DQN采取不一样的网络框架
            self.q_net = VAnet(state_dim, hidden_dim,
                               self.action_dim).to(device)
            self.target_q_net = VAnet(state_dim, hidden_dim,
                                      self.action_dim).to(device)
        else:
            self.q_net = Qnet(state_dim, hidden_dim,
                              self.action_dim).to(device)
            self.target_q_net = Qnet(state_dim, hidden_dim,
                                     self.action_dim).to(device)
        self.optimizer = torch.optim.Adam(self.q_net.parameters(),
                                          lr=learning_rate)
        self.gamma = gamma
        self.epsilon = epsilon
        self.target_update = target_update
        self.count = 0
        self.dqn_type = dqn_type
        self.device = device

    def take_action(self, state):  # state(3,)
        if np.random.random() < self.epsilon:
            # 判断容量是否足够，
            action = np.random.randint(self.action_dim)
        else:
            state = torch.tensor(state, dtype=torch.float).to(self.device)
            state = state.view(1, 21)
            action = self.q_net(state).argmax().item()
        return action

    def max_q_value(self, state):  # state:（3,）
        state = state.view(1, 21)
        state = torch.tensor(state, dtype=torch.float).to(self.device)
        return self.q_net(state).max().item()

    def update(self, transition_dict):
        """

        :param transition_dict:
        :param states: 64,3
        :param actions: 64
        :param next_states: 64,3
        :param rewards: 64
        :param dones: 64
        :return:
        """
        states = torch.tensor(transition_dict['states'],
                              dtype=torch.float).to(self.device)
        actions = torch.tensor(transition_dict['actions']).view(-1, 1).to(
            self.device)
        rewards = torch.tensor(transition_dict['rewards'],
                               dtype=torch.float).view(-1, 1).to(self.device)
        next_states = torch.tensor(transition_dict['next_states'],
                                   dtype=torch.float).to(self.device)
        dones = torch.tensor(transition_dict['dones'],
                             dtype=torch.float).view(-1, 1).to(self.device)
        # print('---------------')
        # print(states.shape)
        # print(next_states.shape)
        # print(rewards.shape)
        # print(actions.shape)
        # print(dones.shape)
        # print('---------------')
        for vec_num in range(3):
            states_now = states[:, :, vec_num:vec_num + 1, :].squeeze(1).squeeze(1)
            next_states_now = next_states[:, :, vec_num:vec_num + 1, :].squeeze(1).squeeze(1)
            # print('---------------')
            # print(states_now.shape)
            # print(next_states_now.shape)
            # print(rewards.shape)
            # print(actions.shape)
            # print(dones.shape)
            # print('---------------')
            actions = actions.squeeze(1)
            actions = torch.tensor(actions)
            # actions = torch.cat([actions,[0]],dim=1)
            # actions = torch.cat([actions,torch.tensor([0])],dim=0)
            # actions = actions.squeeze()
            # actions = actions[:63]
            actions = actions.view(-1, 1)
            dones = dones.view(-1, 1)
            # print(rewards.shape)
            rewards = rewards.view(-1, 1)
            # print('states now : ',states_now.shape)
            # print('actions : ',actions.shape)
            q_values = self.q_net(states_now).gather(1, actions)  # q_values (64, 1)
            # print('q values ',q_values.shape)
            if self.dqn_type == 'DoubleDQN':
                max_action = self.q_net(next_states_now).max(1)[1].view(-1, 1)
                max_next_q_values = self.target_q_net(next_states).gather(
                    1, max_action)
            else:
                max_next_q_values = self.target_q_net(next_states_now.squeeze(1)).max(1)[0].view(
                    -1, 1)  # max_next_q_values (64 , 1)
            # print(rewards.shape)
            # print(max_next_q_values.shape)
            # rewards = torch.cat([rewards,torch.tensor([0])])
            # print(rewards.shape)
            # rewards = rewards[:63]
            # rewards = rewards.view(-1,1)
            max_next_q_values = max_next_q_values.squeeze(1)
            # print(max_next_q_values.shape)

            # max_next_q_values = torch.cat([max_next_q_values,torch.tensor([0])])
            # print(rewards.shape,max_next_q_values.shape)
            # print(dones.shape)

            # 执行逐元素操作
            q_targets = rewards + self.gamma * max_next_q_values * (1 - dones)  # q_targets 的形状为 [63, 1]
            # q_targets = rewards + self.gamma * max_next_q_values * (1 - dones)  # q_targets(64, 1)
            # print('----------')
            # print(q_values.shape,q_targets.shape)
            dqn_loss = torch.mean(F.mse_loss(q_values, q_targets))
            self.optimizer.zero_grad()
            dqn_loss.backward()
            self.optimizer.step()

        if self.count % self.target_update == 0:
            self.target_q_net.load_state_dict(self.q_net.state_dict())
        self.count += 1


lr = 2e-2
num_episodes = 1000
hidden_dim = 128
gamma = 0.98
epsilon = 0.01
target_update = 50
buffer_size = 5000
minimal_size = 1000
batch_size = 64
device = torch.device('cpu')

# env_name = 'Pendulum-v1'
# env = gym.make(env_name)
env = env.Env()
# state_dim = env.observation_space.shape[0]
state_dim = 21
# action_dim = 11  # 将连续动作分成11个离散动作
# vec_num = 3
# node_num = 21
action_dim = 21

random.seed(0)
np.random.seed(0)
torch.manual_seed(0)
replay_buffer = rl_utils.ReplayBuffer(buffer_size)
agent = DQN(state_dim, hidden_dim, action_dim, lr, gamma, epsilon,
            target_update, device, 'DuelingDQN')
return_list, max_q_value_list = train_DQN(agent, env, num_episodes,
                                          replay_buffer, minimal_size,
                                          batch_size)

episodes_list = list(range(len(return_list)))
mv_return = rl_utils.moving_average(return_list, 5)
plt.plot(episodes_list, mv_return)
plt.xlabel('Episodes')
plt.ylabel('Returns')
plt.title('Dueling DQN')
plt.show()

frames_list = list(range(len(max_q_value_list)))
plt.plot(frames_list, max_q_value_list)
# plt.axhline(0, c='orange', ls='--')
# plt.axhline(10, c='red', ls='--')
plt.xlabel('Frames')
plt.ylabel('Q value')
plt.title('Dueling DQN')
plt.show()
